import logging as log
from pathlib import Path
log.basicConfig(level=log.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')

def remove_duplicates(lst):
    seen = {}
    return [seen.setdefault(x, x) for x in lst if x not in seen]


if __name__ == "__main__":
    files = Path("D:\stopWords").glob("*.txt")
    stop_list = []
    for file in files:
        with open(file, "r", encoding="utf-8") as f:
            stop_list.extend(f.read().split())
    # 去重,要求保证原顺序
    stop_list = remove_duplicates(stop_list)
    for i in range(len(stop_list)):
        log.info(f"{i+1} {stop_list[i]}")
    with open("D:\stopWords\stopWords-all.txt", "w", encoding="utf-8") as f:
        f.write("\n".join(stop_list))
